from bs4 import BeautifulSoup
import urllib2
import datetime
import time
import random
import os
import locale

import sys
reload(sys)
sys.setdefaultencoding('utf8')


locale.setlocale(locale.LC_ALL,'nl_NL')		
def convert_date(mydate):
	return datetime.datetime.strptime(mydate.replace(' ',''), '%A%d%B%Y').strftime('%m-%d-%y')
	
def convert_date2(mydate):
	return datetime.datetime.strptime(mydate.replace(' ',''), '%d%B%Y').strftime('%m-%d-%y')

def process_votelist(votelist):
	votelist_date=votelist.split('/')[-1].replace('.html','')
	print votelist_date
	votelist_presoup=open(votelist).read()
	votelist_soup=BeautifulSoup(votelist_presoup,'html.parser')
	try:
		votes=votelist_soup.find_all('ul',{'class':u'search-result-list reset'})[0].find_all('li')
		for vote in votelist_soup.find_all('ul',{'class':u'search-result-list reset'})[0].find_all('li'):
			votedict={}
			try: votedict['name']=vote.h3.string 
			except: pass
			try: votedict['result']=vote.find_all('p',{'class':'result'})[0].span.string 
			except: pass
			try:
				basic_info=vote.find_all('div',{'class':'search-result-properties'})[0] 
				votedict['number']=basic_info.p.string
				votedict['date']=basic_info.find_all('p',{'class':'date'})[0]
			except: pass
			try: votedict['method']=vote.find_all('p',{'class':'vote-type'})[0].span.string
			except: pass
			try: votedict['category']=vote.find_all('p',{'class':'search-result-category'})[0].string
			except: pass
			if votedict!={}:
				votedict['file']=votelist_date
				votedicts.append(votedict)
	except:
		problems.append(votelist_date)

votedicts=[]
problems=[]

for file in os.listdir('/Users/bde254/Desktop/NL-TweedeKamer-Votes/'):
	process_votelist('/Users/bde254/Desktop/NL-TweedeKamer-Votes/'+file)

columnames=['file','name','result','number','date','method','category']


for votedict in votedicts:
	for key in columnames:
		if key in votedict.keys():
			if votedict[key]!= None:
				votedict[key]=votedict[key].encode('utf-8')
			else:
				votedict[key]='NA'
		else:
			votedict[key]="NA"

with open('/Users/bde254/Desktop/NL-TweedeKamer-Votes/NL-Votes.csv', 'wb') as f:
	my_writer = csv.DictWriter(f, fieldnames=votedicts[0].keys())
  	my_writer.writeheader()
	for votedict in votedicts:
		if votedict['method']!='NA':
			my_writer.writerow(votedict)